bitkeeper revision 1.1628 (429dc9b7MTwsBkscbFS1sK8SbwJhdg)

Fix domain shutdown so that the new status, and notification to domain0,
occur *after* the domain is fully descheduled and its execution state
synchronised.
Signed-off-by: Keir Fraser <keir@xensource.com>

diff --git a/tools/gdb/gdb-6.2.1-xen-sparse/gdb/gdbserver/linux-xen-low.c b/tools/gdb/gdb-6.2.1-xen-sparse/gdb/gdbserver/linux-xen-low.c
index e2c72f8798d3fb8649ec18acbec0741f411ea8c8..015c4f19380d9c39613ea6f9ecf91ab4e9b4a054 100644 (file)
--- a/tools/gdb/gdb-6.2.1-xen-sparse/gdb/gdbserver/linux-xen-low.c
+++ b/tools/gdb/gdb-6.2.1-xen-sparse/gdb/gdbserver/linux-xen-low.c
@@ -42,8 +42,7 @@ int (*myxcwait)(int domain, int *status, int options) ;
 
 
 #define DOMFLAGS_DYING     (1<<0) /* Domain is scheduled to die.             */
-#define DOMFLAGS_CRASHED   (1<<1) /* Crashed domain; frozen for postmortem.  */
-#define DOMFLAGS_SHUTDOWN  (1<<2) /* The guest OS has shut itself down.      */
+#define DOMFLAGS_SHUTDOWN  (1<<2) /* The guest OS has shut down.             */
 #define DOMFLAGS_PAUSED    (1<<3) /* Currently paused by control software.   */
 #define DOMFLAGS_BLOCKED   (1<<4) /* Currently blocked pending an event.     */
 #define DOMFLAGS_RUNNING   (1<<5) /* Domain is currently running.            */
@@ -220,7 +219,7 @@ linux_wait (char *status)
   if (myxcwait(current_domain, &w, 0))
       return -1;
   
-  if (w & (DOMFLAGS_CRASHED|DOMFLAGS_DYING)) {
+  if (w & (DOMFLAGS_SHUTDOWN|DOMFLAGS_DYING)) {
       *status = 'W';
       return 0;
   }

diff --git a/tools/libxc/xc_domain.c b/tools/libxc/xc_domain.c
index b51c1c608894fba5b292ee3fe3087aad2ffb6d52..8f0bba3216e5daa1f21a03037b1eefc2908250f0 100644 (file)
--- a/tools/libxc/xc_domain.c
+++ b/tools/libxc/xc_domain.c
@@ -86,7 +86,6 @@ int xc_domain_getinfo(int xc_handle,
         info->domid      = (u16)op.u.getdomaininfo.domain;
 
         info->dying    = !!(op.u.getdomaininfo.flags & DOMFLAGS_DYING);
-        info->crashed  = !!(op.u.getdomaininfo.flags & DOMFLAGS_CRASHED);
         info->shutdown = !!(op.u.getdomaininfo.flags & DOMFLAGS_SHUTDOWN);
         info->paused   = !!(op.u.getdomaininfo.flags & DOMFLAGS_PAUSED);
         info->blocked  = !!(op.u.getdomaininfo.flags & DOMFLAGS_BLOCKED);
@@ -96,6 +95,12 @@ int xc_domain_getinfo(int xc_handle,
             (op.u.getdomaininfo.flags>>DOMFLAGS_SHUTDOWNSHIFT) & 
             DOMFLAGS_SHUTDOWNMASK;
 
+        if ( info->shutdown && (info->shutdown_reason == SHUTDOWN_crash) )
+        {
+            info->shutdown = 0;
+            info->crashed  = 1;
+        }
+
         info->nr_pages = op.u.getdomaininfo.tot_pages;
         info->max_memkb = op.u.getdomaininfo.max_pages<<(PAGE_SHIFT);
         info->shared_info_frame = op.u.getdomaininfo.shared_info_frame;

diff --git a/xen/arch/ia64/xenmisc.c b/xen/arch/ia64/xenmisc.c
index c4630bd7fd9ff6f78c06122d3de0cd64c13b74b4..e82dd8482d7fc30105f862defb06edc7984beb80 100644 (file)
--- a/xen/arch/ia64/xenmisc.c
+++ b/xen/arch/ia64/xenmisc.c
@@ -304,7 +304,8 @@ loop:
        printf(buf);
        if (regs) show_registers(regs);
        domain_pause_by_systemcontroller(current->domain);
-       set_bit(_DOMF_crashed, ed->domain->domain_flags);
+       ed->domain->shutdown_code = SHUTDOWN_crash;
+       set_bit(_DOMF_shutdown, ed->domain->domain_flags);
        if (ed->domain->domain_id == 0) {
                int i = 1000000000L;
                // if domain0 crashes, just periodically print out panic

diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c
index 872fa49e5f7a1dd4cbefed09d87c0af2eb81eb07..8a60d435ee8b6707c9893475acea44482e678edd 100644 (file)
--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -2971,6 +2971,24 @@ void ptwr_destroy(struct domain *d)
     free_xenheap_page((unsigned long)d->arch.ptwr[PTWR_PT_INACTIVE].page);
 }
 
+void cleanup_writable_pagetable(struct domain *d)
+{
+    if ( unlikely(!VM_ASSIST(d, VMASST_TYPE_writable_pagetables)) )
+        return;
+
+    if ( unlikely(shadow_mode_enabled(d)) )
+    {
+        shadow_sync_all(d);
+    }
+    else
+    {
+        if ( d->arch.ptwr[PTWR_PT_ACTIVE].l1va )
+            ptwr_flush(d, PTWR_PT_ACTIVE);
+        if ( d->arch.ptwr[PTWR_PT_INACTIVE].l1va )
+            ptwr_flush(d, PTWR_PT_INACTIVE);
+    }
+}
+
 int map_pages_to_xen(
     unsigned long virt,
     unsigned long pfn,

diff --git a/xen/common/dom0_ops.c b/xen/common/dom0_ops.c
index 1e5bcf5cf81610783920214d6bde70d5d8b1133f..aae2a1172ccf44eabd1ccb8c62b26352552a5cf8 100644 (file)
--- a/xen/common/dom0_ops.c
+++ b/xen/common/dom0_ops.c
@@ -353,7 +353,6 @@ long do_dom0_op(dom0_op_t *u_dom0_op)
 
         op->u.getdomaininfo.flags = flags |
             ((d->domain_flags & DOMF_dying)    ? DOMFLAGS_DYING    : 0) |
-            ((d->domain_flags & DOMF_crashed)  ? DOMFLAGS_CRASHED  : 0) |
             ((d->domain_flags & DOMF_shutdown) ? DOMFLAGS_SHUTDOWN : 0) |
             d->shutdown_code << DOMFLAGS_SHUTDOWNSHIFT;
 

diff --git a/xen/common/domain.c b/xen/common/domain.c
index 709240e202dcc67ec134c68b2b99b17f11185ced..fd32c54ac50d300a87c9aa40f9f56b3206e18987 100644 (file)
--- a/xen/common/domain.c
+++ b/xen/common/domain.c
@@ -134,11 +134,7 @@ void domain_crash(void)
     show_registers(guest_cpu_user_regs());
 #endif
 
-    set_bit(_DOMF_crashed, &d->domain_flags);
-
-    send_guest_virq(dom0->exec_domain[0], VIRQ_DOM_EXC);
-
-    raise_softirq(SCHEDULE_SOFTIRQ);
+    domain_shutdown(SHUTDOWN_crash);
 }
 
 
@@ -150,9 +146,49 @@ void domain_crash_synchronous(void)
 }
 
 
+static struct domain *domain_shuttingdown[NR_CPUS];
+
+static void domain_shutdown_finalise(void)
+{
+    struct domain *d;
+    struct exec_domain *ed;
+
+    d = domain_shuttingdown[smp_processor_id()];
+    domain_shuttingdown[smp_processor_id()] = NULL;
+
+    BUG_ON(d == NULL);
+    BUG_ON(d == current->domain);
+    BUG_ON(!test_bit(_DOMF_shuttingdown, &d->domain_flags));
+    BUG_ON(test_bit(_DOMF_shutdown, &d->domain_flags));
+
+    /* Make sure that every vcpu is descheduled before we finalise. */
+    for_each_exec_domain ( d, ed )
+        while ( test_bit(_VCPUF_running, &ed->vcpu_flags) )
+            cpu_relax();
+
+    sync_lazy_execstate_cpuset(d->cpuset);
+    BUG_ON(d->cpuset != 0);
+
+    sync_pagetable_state(d);
+
+    set_bit(_DOMF_shutdown, &d->domain_flags);
+    clear_bit(_DOMF_shuttingdown, &d->domain_flags);
+
+    send_guest_virq(dom0->exec_domain[0], VIRQ_DOM_EXC);
+}
+
+static __init int domain_shutdown_finaliser_init(void)
+{
+    open_softirq(DOMAIN_SHUTDOWN_FINALISE_SOFTIRQ, domain_shutdown_finalise);
+    return 0;
+}
+__initcall(domain_shutdown_finaliser_init);
+
+
 void domain_shutdown(u8 reason)
 {
     struct domain *d = current->domain;
+    struct exec_domain *ed;
 
     if ( d->domain_id == 0 )
     {
@@ -173,14 +209,18 @@ void domain_shutdown(u8 reason)
         }
     }
 
-    if ( (d->shutdown_code = reason) == SHUTDOWN_crash )
-        set_bit(_DOMF_crashed, &d->domain_flags);
-    else
-        set_bit(_DOMF_shutdown, &d->domain_flags);
-
-    send_guest_virq(dom0->exec_domain[0], VIRQ_DOM_EXC);
+    /* Mark the domain as shutting down. */
+    d->shutdown_code = reason;
+    if ( !test_and_set_bit(_DOMF_shuttingdown, &d->domain_flags) )
+    {
+        /* This vcpu won the race to finalise the shutdown. */
+        domain_shuttingdown[smp_processor_id()] = d;
+        raise_softirq(DOMAIN_SHUTDOWN_FINALISE_SOFTIRQ);
+    }
 
-    raise_softirq(SCHEDULE_SOFTIRQ);
+    /* Put every vcpu to sleep, but don't wait (avoids inter-vcpu deadlock). */
+    for_each_exec_domain ( d, ed )
+        domain_sleep_nosync(ed);
 }
 
 
@@ -190,8 +230,7 @@ void domain_destruct(struct domain *d)
     struct domain **pd;
     atomic_t      old, new;
 
-    if ( !test_bit(_DOMF_dying, &d->domain_flags) )
-        BUG();
+    BUG_ON(!test_bit(_DOMF_dying, &d->domain_flags));
 
     /* May be already destructed, or get_domain() can race us. */
     _atomic_set(old, 0);
@@ -225,10 +264,9 @@ void domain_destruct(struct domain *d)
 
 void exec_domain_pause(struct exec_domain *ed)
 {
-    ASSERT(ed != current);
+    BUG_ON(ed == current);
     atomic_inc(&ed->pausecnt);
-    domain_sleep(ed);
-    sync_lazy_execstate_cpuset(ed->domain->cpuset & (1UL << ed->processor));
+    domain_sleep_sync(ed);
 }
 
 void domain_pause(struct domain *d)
@@ -237,17 +275,15 @@ void domain_pause(struct domain *d)
 
     for_each_exec_domain( d, ed )
     {
-        ASSERT(ed != current);
+        BUG_ON(ed == current);
         atomic_inc(&ed->pausecnt);
-        domain_sleep(ed);
+        domain_sleep_sync(ed);
     }
-
-    sync_lazy_execstate_cpuset(d->cpuset);
 }
 
 void exec_domain_unpause(struct exec_domain *ed)
 {
-    ASSERT(ed != current);
+    BUG_ON(ed == current);
     if ( atomic_dec_and_test(&ed->pausecnt) )
         domain_wake(ed);
 }
@@ -266,12 +302,10 @@ void domain_pause_by_systemcontroller(struct domain *d)
 
     for_each_exec_domain ( d, ed )
     {
-        ASSERT(ed != current);
+        BUG_ON(ed == current);
         if ( !test_and_set_bit(_VCPUF_ctrl_pause, &ed->vcpu_flags) )
-            domain_sleep(ed);
+            domain_sleep_sync(ed);
     }
-
-    sync_lazy_execstate_cpuset(d->cpuset);
 }
 
 void domain_unpause_by_systemcontroller(struct domain *d)

diff --git a/xen/common/schedule.c b/xen/common/schedule.c
index 1e5df627d1a34125a6f61880905510338e7c17aa..7707ea03af9649b49bd92d4a9543254925a59b8b 100644 (file)
--- a/xen/common/schedule.c
+++ b/xen/common/schedule.c
@@ -190,7 +190,7 @@ void sched_rem_domain(struct exec_domain *ed)
     TRACE_2D(TRC_SCHED_DOM_REM, ed->domain->domain_id, ed->vcpu_id);
 }
 
-void domain_sleep(struct exec_domain *ed)
+void domain_sleep_nosync(struct exec_domain *ed)
 {
     unsigned long flags;
 
@@ -200,10 +200,16 @@ void domain_sleep(struct exec_domain *ed)
     spin_unlock_irqrestore(&schedule_data[ed->processor].schedule_lock, flags);
 
     TRACE_2D(TRC_SCHED_SLEEP, ed->domain->domain_id, ed->vcpu_id);
- 
-    /* Synchronous. */
+} 
+
+void domain_sleep_sync(struct exec_domain *ed)
+{
+    domain_sleep_nosync(ed);
+
     while ( test_bit(_VCPUF_running, &ed->vcpu_flags) && !domain_runnable(ed) )
         cpu_relax();
+
+    sync_lazy_execstate_cpuset(ed->domain->cpuset & (1UL << ed->processor));
 }
 
 void domain_wake(struct exec_domain *ed)

diff --git a/xen/include/asm-x86/mm.h b/xen/include/asm-x86/mm.h
index ac0d3cd40b7917613722d71b23988f5de7b85782..5c7f91f127ac01543fe65d486bd969f544c6cd3f 100644 (file)
--- a/xen/include/asm-x86/mm.h
+++ b/xen/include/asm-x86/mm.h
@@ -314,21 +314,8 @@ void ptwr_flush(struct domain *, const int);
 int  ptwr_do_page_fault(struct domain *, unsigned long);
 int  revalidate_l1(struct domain *, l1_pgentry_t *, l1_pgentry_t *);
 
-#define cleanup_writable_pagetable(_d)                                      \
-    do {                                                                    \
-        if ( likely(VM_ASSIST((_d), VMASST_TYPE_writable_pagetables)) )     \
-        {                                                                   \
-            if ( likely(!shadow_mode_enabled(_d)) )                         \
-            {                                                               \
-                if ( (_d)->arch.ptwr[PTWR_PT_ACTIVE].l1va )                 \
-                    ptwr_flush((_d), PTWR_PT_ACTIVE);                       \
-                if ( (_d)->arch.ptwr[PTWR_PT_INACTIVE].l1va )               \
-                    ptwr_flush((_d), PTWR_PT_INACTIVE);                     \
-            }                                                               \
-            else                                                            \
-                shadow_sync_all(_d);                                        \
-        }                                                                   \
-    } while ( 0 )
+void cleanup_writable_pagetable(struct domain *d);
+#define sync_pagetable_state(d) cleanup_writable_pagetable(d)
 
 int audit_adjust_pgtables(struct domain *d, int dir, int noisy);
 

diff --git a/xen/include/public/dom0_ops.h b/xen/include/public/dom0_ops.h
index 76bd18e079ae18eb92ed2ff9eb8a17c54d08c2d1..4c3d5271a2f8b52da31268127fa8bedf9ca8b354 100644 (file)
--- a/xen/include/public/dom0_ops.h
+++ b/xen/include/public/dom0_ops.h
@@ -72,8 +72,7 @@ typedef struct {
     domid_t  domain;                  /* NB. IN/OUT variable. */
     /* OUT variables. */
 #define DOMFLAGS_DYING     (1<<0) /* Domain is scheduled to die.             */
-#define DOMFLAGS_CRASHED   (1<<1) /* Crashed domain; frozen for postmortem.  */
-#define DOMFLAGS_SHUTDOWN  (1<<2) /* The guest OS has shut itself down.      */
+#define DOMFLAGS_SHUTDOWN  (1<<2) /* The guest OS has shut down.             */
 #define DOMFLAGS_PAUSED    (1<<3) /* Currently paused by control software.   */
 #define DOMFLAGS_BLOCKED   (1<<4) /* Currently blocked pending an event.     */
 #define DOMFLAGS_RUNNING   (1<<5) /* Domain is currently running.            */

diff --git a/xen/include/xen/mm.h b/xen/include/xen/mm.h
index 807987045f0d725150285878edfc1ce7c618cddc..4e7f57064398b66d4dc20b2d149b7076cf8f8c87 100644 (file)
--- a/xen/include/xen/mm.h
+++ b/xen/include/xen/mm.h
@@ -48,4 +48,8 @@ extern struct list_head page_scrub_list;
 
 #include <asm/mm.h>
 
+#ifndef sync_pagetable_state
+#define sync_pagetable_state(d) ((void)0)
+#endif
+
 #endif /* __XEN_MM_H__ */

diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h
index 0462fda5c1011f516594879b9b1e139fa5ae3d91..e1af30ed2d80154824d87e939dc53ca0ed6ee791 100644 (file)
--- a/xen/include/xen/sched.h
+++ b/xen/include/xen/sched.h
@@ -255,7 +255,8 @@ long sched_ctl(struct sched_ctl_cmd *);
 long sched_adjdom(struct sched_adjdom_cmd *);
 int  sched_id();
 void domain_wake(struct exec_domain *d);
-void domain_sleep(struct exec_domain *d);
+void domain_sleep_nosync(struct exec_domain *d);
+void domain_sleep_sync(struct exec_domain *d);
 
 /*
  * Force loading of currently-executing domain state on the specified set
@@ -375,9 +376,9 @@ extern struct domain *domain_list;
  /* Guest shut itself down for some reason. */
 #define _DOMF_shutdown         4
 #define DOMF_shutdown          (1UL<<_DOMF_shutdown)
- /* Domain has crashed and cannot continue to execute. */
-#define _DOMF_crashed          5
-#define DOMF_crashed           (1UL<<_DOMF_crashed)
+ /* Guest is in process of shutting itself down (becomes DOMF_shutdown). */
+#define _DOMF_shuttingdown     5
+#define DOMF_shuttingdown      (1UL<<_DOMF_shuttingdown)
  /* Death rattle. */
 #define _DOMF_dying            6
 #define DOMF_dying             (1UL<<_DOMF_dying)
@@ -386,7 +387,7 @@ static inline int domain_runnable(struct exec_domain *ed)
 {
     return ( (atomic_read(&ed->pausecnt) == 0) &&
              !(ed->vcpu_flags & (VCPUF_blocked|VCPUF_ctrl_pause)) &&
-             !(ed->domain->domain_flags & (DOMF_shutdown|DOMF_crashed)) );
+             !(ed->domain->domain_flags & (DOMF_shutdown|DOMF_shuttingdown)) );
 }
 
 void exec_domain_pause(struct exec_domain *ed);

diff --git a/xen/include/xen/softirq.h b/xen/include/xen/softirq.h
index a538540247b0de7bc6cd92db32f22f2814f64a8a..de3480482ae2edd62dbe9689050263b0758cdb17 100644 (file)
--- a/xen/include/xen/softirq.h
+++ b/xen/include/xen/softirq.h
@@ -8,7 +8,8 @@
 #define KEYPRESS_SOFTIRQ                  3
 #define NMI_SOFTIRQ                       4
 #define PAGE_SCRUB_SOFTIRQ                5
-#define NR_SOFTIRQS                       6
+#define DOMAIN_SHUTDOWN_FINALISE_SOFTIRQ  6
+#define NR_SOFTIRQS                       7
 
 #ifndef __ASSEMBLY__